#!/usr/bin/env python
# -*- coding:utf-8 -*-
# @FileName  :huaban_pins.py
# @Time      :2023/7/27 
# @Author    :CL
# @email     :1037654919@qq.com

import os
import json
import random
from tqdm import tqdm
from utils import MongoDBUtil, get_html, mongo_manager
from multiprocessing import Pool
import requests
from huaban_util import Huaban

MongoDBUtil = MongoDBUtil()
huaban_boards = mongo_manager("huaban_boards", db="car_images")
huaban = Huaban()

proxies = {'http': '127.0.0.1:15732',
           'https': '127.0.0.1:15732'}


def get_proxy():
    port = random.randint(24000, 24400)
    proxies = {'http': f'http://zheng123:zheng123@haproxy.iinti.cn:{port}',
               'https': f'http://zheng123:zheng123@haproxy.iinti.cn:{port}'}
    return proxies


def insert_data(tablename, datas):
    if len(datas) > 0 and datas is not None:
        for data in datas:
            # data['_id'] = data['id']
            try:
                # print(data)
                MongoDBUtil.insert_one(tablename, data)
            except Exception as e:
                print(e)
    else:
        pass


def save_summary(path, file, jsondata):
    with open(path + file, 'w') as f:
        f.write(json.dumps(jsondata,ensure_ascii=False)) # str 转换成JSON字符串时，中文字符会被转义,使用 ensure_ascii=False显示中文
    f.close()


def save_txt(txt_filename, data, PWD):
    with open(PWD + txt_filename, 'w') as f:
        f.write(data)
    f.close()


def save_image(name, url, PWD):
    print('save image:', url)
    # name = url.split('/')[-1]
    # 将图片以最后的数字+后缀命名，方便检查文件是否存在
    filename = PWD + name
    if os.path.isfile(filename):  # 如果文件已爬取，则不再重复爬取
        print("文件存在：", filename)
        return 2
    # urllib.request.urlretrieve(url, filename=path+filename)
    try:
        response = requests.get(url, proxies=get_proxy(), stream=True, timeout=30)
        with open(filename, 'wb') as fd:
            fd.write(response.content)
        requests.session().close()
        return 1
    except  Exception as e:
        print(url, '保存图片失败', e)
        return 0


def parse_url(seed):
    huaban_boards = mongo_manager("huaban_boards", db="car_images")
    kw = seed['keyword']
    board_id = seed['_id']
    title = seed['keyword_e']
    pin_count = seed['pin_count']
    url = "https://api.huaban.com/boards/{}/pins".format(board_id)
    print('begin 访问网页:', url, '  pin_count:', pin_count)
    # 创建目录文件夹
    file = f'huaban/{kw}/'
    if os.path.exists(file):
        pass
    else:
        os.mkdir(file)
    # 创建内容文件夹，如果有，表明已经爬取
    file = f'huaban/{kw}/{board_id}/'
    if os.path.exists(file):
        name = os.listdir(file)
        if len(name) > 1 and 'summary.json' in name:
            print('文件夹已经存在,文件已经爬取')
            seed['status'] = 'success'
            huaban_boards.updateOne({'_id': seed['_id']}, seed)
            huaban_boards.close()
            return
    else:
        os.mkdir(file)
    response = huaban.get_pins(board_id, maxs=None)
    pins = response['pins']
    board = response['board']
    desc = board['description']
    category_id = board['category_id']
    last_pin_id = None
    success_count = 0
    images = []
    flag = 40
    while flag>=40:
        try:
            response = huaban.get_pins(board_id, maxs=last_pin_id)
            if response:
                pins = response['pins']
                flag = len(pins)
                last_pin_id = pins[-1]['pin_id']
                for pin in pins:
                    # print(pin['pin_id'])
                    datas = {}
                    datas['tags'] = pin['tags']
                    tag_list = pin['tags']
                    datas['_id'] = pin['pin_id']
                    datas['pin_id'] = pin['pin_id']
                    datas['keyword'] = seed['keyword']
                    datas['board_id'] = pin['board_id']
                    datas['raw_text'] = pin['raw_text']
                    datas['file'] = pin['file']
                    href = 'https://huaban.com/pins/' + str(pin['pin_id'])
                    datas['href'] = href
                    src = 'https://gd-hbimg.huaban.com/' + str(pin['file']['key'])
                    datas['src'] = src
                    try:
                        MongoDBUtil.insert_one('huaban_pins', datas)
                    except Exception as e:
                        print(e)
                        # continue
                    name = src.split('/')[-1]
                    # 保存图片
                    result = save_image(name=name, url=src, PWD=file)
                    # 保存文本
                    if result == 1:
                        txt_filename = name.split('.')[0] + '.txt'
                        description = str(pin['raw_text']).rsplit('|',1)[0]
                        if str(description).strip() == '':
                            description = desc
                        txt_data = str(title) + ',' + str(description)
                        for tag in tag_list:
                            txt_data += ',' + str(tag)
                        save_txt(txt_filename=txt_filename, data=txt_data, PWD=file)
                        images.append(
                            {
                                "img_file": name,
                                "label_file": txt_filename,
                                "tag": tag_list,
                                "title": title,
                                "description": str(description)
                            })
                        success_count += 1
                        datas['status'] = 'success'
                        huaban_pins = mongo_manager("huaban_pins", db="car_images")
                        huaban_pins.updateOne({'_id': datas['_id']}, datas)
                        huaban_pins.close()
        except Exception as e:
            print(e)
    # 保存save_summary
    seed['success_count'] = success_count
    if success_count > 0:
        seed['description'] = desc
        seed['category_id'] = category_id
        seed['status'] = 'success'
        jsondata = {"page_url": url, "images": images}
        save_summary(path=file, file='summary.json', jsondata=jsondata)
    else:
        seed['status'] = 'unsuccess'
    huaban_boards.updateOne({'_id': seed['_id']}, seed)
    huaban_boards.close()
    return


def main():
    seeds = huaban_boards.findAll({"status": None})
    lists = []
    for seed in seeds:
        lists.append(seed)
    huaban_boards.close()
    print('playlist_song_list None:', len(lists))
    pool = Pool(processes=30)
    pool.map(parse_url, lists)
    pool.close()  # 关闭进程池，不再接受新的进程
    pool.join()


if __name__ == "__main__":
    main()

    # # test
    # seed = {'_id': 43767873, 'board_id': 43767873, 'keyword': 'car', 'page': 1, 'user_id': 19807972, 'title': '# 汽车广告-CAR', 'pin_count': 7626, 'seq': 43767873}
    # parse_url(seed)

    # # 恢复mongodb数据
    # huaban_boards = mongo_manager("huaban_boards", db="car_images")
    # seeds = huaban_boards.findAll({"status": 'success'})
    # lists =[]
    # for seed in seeds:
    #     seed['status'] = None
    #     lists.append(seed)
    #     huaban_boards.updateOne({'_id': seed['_id']}, seed)
    # huaban_boards.close()
    # print(len(lists))